{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "<img src=\"img/self_attn.png\" width=\"200\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "pip3 install tensor2tensor\n",
    "\"\"\"\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensor2tensor.utils import beam_search\n",
    "from tensor2tensor.layers.common_attention import add_timing_signal_1d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'batch_size': 64,\n",
    "    'text_iter_step': 25,\n",
    "    'seq_len': 200,\n",
    "    'hidden_dim': 128,\n",
    "    'num_head': 8,\n",
    "    'n_hidden_layer': 2,\n",
    "    'display_step': 10,\n",
    "    'generate_step': 100,\n",
    "    'beam_size': 5,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_text(file_path):\n",
    "    with open(file_path) as f:\n",
    "        text = f.read()\n",
    "    \n",
    "    char2idx = {c: i+3 for i, c in enumerate(set(text))}\n",
    "    char2idx['<pad>'] = 0\n",
    "    char2idx['<start>'] = 1\n",
    "    char2idx['<end>'] = 2\n",
    "    \n",
    "    ints = np.array([char2idx[char] for char in list(text)])\n",
    "    return ints, char2idx\n",
    "\n",
    "def next_batch(ints):\n",
    "    len_win = params['seq_len'] * params['batch_size']\n",
    "    for i in range(0, len(ints)-len_win, params['text_iter_step']):\n",
    "        clip = ints[i: i+len_win]\n",
    "        yield clip.reshape([params['batch_size'], params['seq_len']])\n",
    "        \n",
    "def input_fn(ints):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: next_batch(ints), tf.int32, tf.TensorShape([None, params['seq_len']]))\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    return iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def start_sent(x):\n",
    "    _x = tf.fill([tf.shape(x)[0], 1], params['char2idx']['<start>'])\n",
    "    return tf.concat([_x, x], 1)\n",
    "\n",
    "def end_sent(x):\n",
    "    _x = tf.fill([tf.shape(x)[0], 1], params['char2idx']['<end>'])\n",
    "    return tf.concat([x, _x], 1)\n",
    "\n",
    "def embed_seq(x, vocab_sz, embed_dim, zero_pad=False, scale=False):\n",
    "    embedding = tf.get_variable('word2vec', [vocab_sz, embed_dim])\n",
    "    if zero_pad:\n",
    "        embedding = tf.concat([tf.zeros([1, embed_dim]), embedding[1:, :]], 0)\n",
    "    x = tf.nn.embedding_lookup(embedding, x)\n",
    "    if scale:\n",
    "        x = x * tf.sqrt(tf.to_float(embed_dim))\n",
    "    return x\n",
    "\n",
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) * (tf.rsqrt(variance + epsilon))\n",
    "    \n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    \n",
    "    return gamma * normalized + beta\n",
    "\n",
    "def self_attention(inputs, is_training, activation=None):\n",
    "    num_units = params['hidden_dim']\n",
    "    num_heads = params['num_head']\n",
    "    T_q = T_k = tf.shape(inputs)[1]\n",
    "\n",
    "    Q_K_V = tf.layers.dense(inputs, 3*num_units, activation)\n",
    "    Q, K, V = tf.split(Q_K_V, 3, -1)\n",
    "    Q_ = tf.concat(tf.split(Q, num_heads, axis=2), 0)                         \n",
    "    K_ = tf.concat(tf.split(K, num_heads, axis=2), 0)                        \n",
    "    V_ = tf.concat(tf.split(V, num_heads, axis=2), 0)                         \n",
    "\n",
    "    align = tf.matmul(Q_, K_, transpose_b=True)                               \n",
    "    align *= tf.rsqrt(tf.to_float(K_.get_shape()[-1].value))\n",
    "\n",
    "    paddings = tf.fill(tf.shape(align), float('-inf'))         \n",
    "    lower_tri = tf.ones([T_q, T_k])                                                \n",
    "    lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()      \n",
    "    masks = tf.tile(tf.expand_dims(lower_tri,0), [tf.shape(align)[0],1,1])       \n",
    "    align = tf.where(tf.equal(masks, 0), paddings, align)               \n",
    "\n",
    "    align = tf.nn.softmax(align)                                                  \n",
    "    align = tf.layers.dropout(align, 0.1, training=is_training)           \n",
    "    x = tf.matmul(align, V_)                                                 \n",
    "    x = tf.concat(tf.split(x, num_heads, axis=0), 2)              \n",
    "    x += inputs                                                                \n",
    "    x = layer_norm(x)                                                 \n",
    "    return x\n",
    "\n",
    "def ffn(inputs, activation=tf.nn.relu):\n",
    "    x = tf.layers.conv1d(inputs, 4*params['hidden_dim'], 1, activation=activation)\n",
    "    x = tf.layers.conv1d(x, params['hidden_dim'], 1, activation=None)\n",
    "    x += inputs\n",
    "    x = layer_norm(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(inputs, reuse, is_training):\n",
    "    with tf.variable_scope('model', reuse=reuse):\n",
    "        x = embed_seq(inputs,\n",
    "                      params['vocab_size'],\n",
    "                      params['hidden_dim'],\n",
    "                      zero_pad=True,\n",
    "                      scale=True)\n",
    "        x = add_timing_signal_1d(x)\n",
    "        x = tf.layers.dropout(x, 0.1, training=is_training)\n",
    "        \n",
    "        for i in range(params['n_hidden_layer']):\n",
    "            with tf.variable_scope('attn_%d'%i, reuse=reuse):\n",
    "                x = self_attention(x, is_training)\n",
    "            with tf.variable_scope('ffn_%d'%i, reuse=reuse):\n",
    "                x = ffn(x)\n",
    "        \n",
    "        logits = tf.layers.dense(x, params['vocab_size'])\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def beam_search_decoding():\n",
    "    batch_size = 1\n",
    "    initial_ids = tf.constant(params['char2idx']['<start>'], tf.int32, [batch_size])\n",
    "    \n",
    "    def symbols_to_logits(ids):\n",
    "        logits = forward(ids, reuse=True, is_training=False)\n",
    "        return logits[:, tf.shape(ids)[1]-1, :]\n",
    "    \n",
    "    final_ids, final_probs = beam_search.beam_search(\n",
    "        symbols_to_logits,\n",
    "        initial_ids,\n",
    "        params['beam_size'],\n",
    "        2 * params['seq_len'],\n",
    "        params['vocab_size'],\n",
    "        0.0,\n",
    "        eos_id = params['char2idx']['<end>'])\n",
    "    \n",
    "    return final_ids[0, 0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_graph():\n",
    "    ints, params['char2idx'] = parse_text('../temp/anna.txt')\n",
    "    params['vocab_size'] = len(params['char2idx'])\n",
    "    params['idx2char'] = {i: c for c, i in params['char2idx'].items()}\n",
    "    print('Vocabulary size:', params['vocab_size'])\n",
    "    X = input_fn(ints)\n",
    "    logits = forward(start_sent(X), reuse=False, is_training=True)\n",
    "    ops = {}\n",
    "    ops['global_step'] = tf.Variable(0, trainable=False)\n",
    "    targets = end_sent(X)\n",
    "    ops['loss'] = tf.reduce_mean(tf.contrib.seq2seq.sequence_loss(\n",
    "        logits = logits,\n",
    "        targets = targets,\n",
    "        weights = tf.to_float(tf.ones_like(targets))))\n",
    "    ops['train'] = tf.train.AdamOptimizer().minimize(ops['loss'],\n",
    "                                                     global_step=ops['global_step'])\n",
    "    ops['generate'] = beam_search_decoding()\n",
    "    return ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    ops = build_graph()\n",
    "    sess = tf.Session()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    while True:\n",
    "        try:\n",
    "            _, step, loss = sess.run([ops['train'], ops['global_step'], ops['loss']])\n",
    "        except tf.errors.OutOfRangeError:\n",
    "            break\n",
    "        else:\n",
    "            if step % params['display_step'] == 0 or step == 1:\n",
    "                print(\"Step %d | Loss %.3f\" % (step, loss))\n",
    "            if step % params['generate_step'] == 0 and step > 1:\n",
    "                ints = sess.run(ops['generate'])\n",
    "                print('\\n'+''.join([params['idx2char'][i] for i in ints])+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size: 86\n",
      "Step 1 | Loss 5.169\n",
      "Step 10 | Loss 2.878\n",
      "Step 20 | Loss 2.671\n",
      "Step 30 | Loss 2.580\n",
      "Step 40 | Loss 2.532\n",
      "Step 50 | Loss 2.488\n",
      "Step 60 | Loss 2.455\n",
      "Step 70 | Loss 2.431\n",
      "Step 80 | Loss 2.407\n",
      "Step 90 | Loss 2.395\n",
      "Step 100 | Loss 2.375\n",
      "\n",
      "<start> his his his his his his his his his his his his his he his he his his his he his his his his his his he his his his his his he his his his his he his his his his his he he his his he his his he he h<end><pad>\n",
      "\n",
      "Step 110 | Loss 2.355\n",
      "Step 120 | Loss 2.337\n",
      "Step 130 | Loss 2.311\n",
      "Step 140 | Loss 2.276\n",
      "Step 150 | Loss 2.253\n",
      "Step 160 | Loss 2.223\n",
      "Step 170 | Loss 2.208\n",
      "Step 180 | Loss 2.180\n",
      "Step 190 | Loss 2.160\n",
      "Step 200 | Loss 2.122\n",
      "\n",
      "<start> his his his his his his his his hise his his his his his his his his his his hise his his hise his his his his his his hise his his his his his his cofe his his his his his his hise his his his hise <end>\n",
      "\n",
      "Step 210 | Loss 2.096\n",
      "Step 220 | Loss 2.060\n",
      "Step 230 | Loss 2.044\n",
      "Step 240 | Loss 2.019\n",
      "Step 250 | Loss 1.991\n",
      "Step 260 | Loss 1.952\n",
      "Step 270 | Loss 1.913\n",
      "Step 280 | Loss 1.896\n",
      "Step 290 | Loss 1.866\n",
      "Step 300 | Loss 1.840\n",
      "\n",
      "<start> and the wat the was fonderd the he\n",
      "moffff the was he his his his his fe.\n",
      "\n",
      "\n",
      "Stepan Arkadyevitch was was as asond tond therend the of he his his\n",
      "coffffffe, he, and he, and and the he he wat he was his <end>\n",
      "\n",
      "Step 310 | Loss 1.808\n",
      "Step 320 | Loss 1.775\n",
      "Step 330 | Loss 1.764\n",
      "Step 340 | Loss 1.732\n",
      "Step 350 | Loss 1.688\n",
      "Step 360 | Loss 1.679\n",
      "Step 370 | Loss 1.661\n",
      "Step 380 | Loss 1.624\n",
      "Step 390 | Loss 1.601\n",
      "Step 400 | Loss 1.586\n",
      "\n",
      "<start> the was wing hims\n",
      "wife.\n",
      "\n",
      "\"Welll, hand, Matvey?\" Stepan Arkadyevitch, and the and gother, and\n",
      "sing ther was the for.\n",
      "\n",
      "\"Darya Alexandrovna?\" Alexanandrovnana, anana, the gought gought gought his conis <end><pad>\n",
      "\n",
      "Step 410 | Loss 1.572\n",
      "Step 420 | Loss 1.554\n",
      "Step 430 | Loss 1.518\n",
      "Step 440 | Loss 1.490\n",
      "Step 450 | Loss 1.467\n",
      "Step 460 | Loss 1.460\n",
      "Step 470 | Loss 1.440\n",
      "Step 480 | Loss 1.444\n",
      "Step 490 | Loss 1.416\n",
      "Step 500 | Loss 1.384\n",
      "\n",
      "<start>the was and to ther her father's and ther saings acould of nothis ther.\n",
      "\n",
      "\"Matvey, inat that tholinat the girl,\" he she said the the she coup, tonck\n",
      "though wh this the doought ther ther doooor.\n",
      "\n",
      "\n",
      "\"Whel<end>\n",
      "\n",
      "Step 510 | Loss 1.368\n",
      "Step 520 | Loss 1.347\n",
      "Step 530 | Loss 1.335\n",
      "Step 540 | Loss 1.327\n",
      "Step 550 | Loss 1.304\n",
      "Step 560 | Loss 1.290\n",
      "Step 570 | Loss 1.287\n",
      "Step 580 | Loss 1.263\n",
      "Step 590 | Loss 1.246\n",
      "Step 600 | Loss 1.222\n",
      "\n",
      "<start> the could of there her. The liberal\n",
      "party said that in that rather sather ather and hather her anysthe\n",
      "berelooook, ther and ther father.\n",
      "\n",
      "\"God, sait thought thought ht hoyd!\" shought; \"\n",
      "\n",
      "\"Whelll, sho<end>\n",
      "\n",
      "Step 610 | Loss 1.196\n",
      "Step 620 | Loss 1.189\n",
      "Step 630 | Loss 1.165\n",
      "Step 640 | Loss 1.159\n",
      "Step 650 | Loss 1.144\n",
      "Step 660 | Loss 1.119\n",
      "Step 670 | Loss 1.113\n",
      "Step 680 | Loss 1.110\n",
      "Step 690 | Loss 1.087\n",
      "Step 700 | Loss 1.054\n",
      "\n",
      "<start> and face, arand led little give him.\n",
      "\n",
      "\"Dolly!\" he said, said in hereaing. \"The said, said nnot litttle girl in\n",
      "voice to to to his was impossible to to make and to him, and o\n",
      "ther fat father face, whi<end>\n",
      "\n",
      "Step 710 | Loss 1.043\n",
      "Step 720 | Loss 1.030\n",
      "Step 730 | Loss 1.032\n",
      "Step 740 | Loss 1.020\n",
      "Step 750 | Loss 1.019\n",
      "Step 760 | Loss 0.993\n",
      "Step 770 | Loss 0.991\n",
      "Step 780 | Loss 0.985\n",
      "Step 790 | Loss 0.985\n",
      "Step 800 | Loss 0.976\n",
      "\n",
      "<start> and face, arand led and loved my\n",
      "with tou mite your in her father. She was sawas impossible; but it was impossible tose mpossible; it was impossible because she could not get out of the her. Everars <end>\n",
      "\n",
      "Step 810 | Loss 0.962\n",
      "Step 820 | Loss 0.968\n",
      "Step 830 | Loss 0.946\n",
      "Step 840 | Loss 0.950\n",
      "Step 850 | Loss 0.948\n",
      "Step 860 | Loss 0.946\n",
      "Step 870 | Loss 0.926\n",
      "Step 880 | Loss 0.928\n",
      "Step 890 | Loss 0.911\n",
      "Step 900 | Loss 0.909\n",
      "\n",
      "<start> and for ther.\n",
      "\n",
      "\"You tefur the chom children,\" she thought, \" said Matvey, s she slam, rowing ge\n",
      "the doned.\n",
      "\n",
      "Stepan Arkadyevitch was fond of a joke: \"And maybe she will come round! That's\n",
      "a good expre<end>\n",
      "\n",
      "Step 910 | Loss 0.884\n",
      "Step 920 | Loss 0.896\n",
      "Step 930 | Loss 0.885\n",
      "Step 940 | Loss 0.872\n",
      "Step 950 | Loss 0.859\n",
      "Step 960 | Loss 0.871\n",
      "Step 970 | Loss 0.861\n",
      "Step 980 | Loss 0.859\n",
      "Step 990 | Loss 0.855\n",
      "Step 1000 | Loss 0.862\n",
      "\n",
      "<start> and face, and am Matvey scon Pheilimonovna him powith he hir papes, and with a subdued treably in\n",
      "h<end><pad><pad><pad><pad><pad><pad>\n",
      "\n",
      "Step 1010 | Loss 0.837\n",
      "Step 1020 | Loss 0.833\n",
      "Step 1030 | Loss 0.835\n",
      "Step 1040 | Loss 0.830\n",
      "Step 1050 | Loss 0.821\n",
      "Step 1060 | Loss 0.812\n",
      "Step 1070 | Loss 0.815\n",
      "Step 1080 | Loss 0.818\n",
      "Step 1090 | Loss 0.791\n",
      "Step 1100 | Loss 0.812\n",
      "\n",
      "<start> she said, and this children, as\n",
      "his alll had, know werere shis friech his childrenct\n",
      "she shad werent of his chis which his cownd umay, and the drad went of the\n",
      "childrerencerer<end><pad><pad><pad><pad><pad><pad><pad><pad><pad>\n",
      "\n",
      "Step 1110 | Loss 0.791\n",
      "Step 1120 | Loss 0.796\n",
      "Step 1130 | Loss 0.781\n",
      "Step 1140 | Loss 0.783\n",
      "Step 1150 | Loss 0.766\n",
      "Step 1160 | Loss 0.770\n",
      "Step 1170 | Loss 0.766\n",
      "Step 1180 | Loss 0.773\n",
      "Step 1190 | Loss 0.760\n",
      "Step 1200 | Loss 0.753\n",
      "\n",
      "<start> and the doorkeper, but coallings he and\n",
      "roves. \"He you have hangeing one! But has he broken it off with her?\" she\n",
      "thought. \"Can it be he sees her? Why didn't I ask him! No, no,\n",
      "reconciliation is impo<end><pad>\n",
      "\n",
      "Step 1210 | Loss 0.741\n",
      "Step 1220 | Loss 0.757\n",
      "Step 1230 | Loss 0.781\n",
      "Step 1240 | Loss 0.772\n",
      "Step 1250 | Loss 0.765\n",
      "Step 1260 | Loss 0.749\n",
      "Step 1270 | Loss 0.749\n",
      "Step 1280 | Loss 0.749\n",
      "Step 1290 | Loss 0.741\n",
      "Step 1300 | Loss 0.736\n",
      "\n",
      "<start> and therefore\n",
      "was one of the lowest in his class. But in spite of his habitually\n",
      "dissipated mode of life, his inferior grade in the service, and his\n",
      "comparative youth, he occcupied the honorable and <end>\n",
      "\n",
      "Step 1310 | Loss 0.734\n",
      "Step 1320 | Loss 0.737\n",
      "Step 1330 | Loss 0.722\n",
      "Step 1340 | Loss 0.736\n",
      "Step 1350 | Loss 0.720\n",
      "Step 1360 | Loss 0.731\n",
      "Step 1370 | Loss 0.705\n",
      "Step 1380 | Loss 0.717\n",
      "Step 1390 | Loss 0.705\n",
      "Step 1400 | Loss 0.702\n",
      "\n",
      "<start> the board, the\n",
      "began.\n",
      "\n",
      "\"If hey knew,\" he thought, said.\n",
      "\n",
      "Stepan Arkadyevitch. \"To be sure we shall!\" said Nikitin.\n",
      "\n",
      "\"A pretty sharp fellow this Fomin mut be,\" said Grinevitch,\" said Stepan Arkadyevit<end>\n",
      "\n",
      "Step 1410 | Loss 0.705\n",
      "Step 1420 | Loss 0.714\n",
      "Step 1430 | Loss 0.684\n",
      "Step 1440 | Loss 0.689\n",
      "Step 1450 | Loss 0.686\n",
      "Step 1460 | Loss 0.693\n",
      "Step 1470 | Loss 0.683\n",
      "Step 1480 | Loss 0.663\n",
      "Step 1490 | Loss 0.664\n",
      "Step 1500 | Loss 0.673\n",
      "\n",
      "<start> the board, and few the and him friend, and his of his\n",
      "subordinates, he well knew how, with his characteristic tact, too\n",
      "diminish the disagreeable impression made on them. Levin was not a\n",
      "disreputably<end>\n",
      "\n",
      "Step 1510 | Loss 0.643\n",
      "Step 1520 | Loss 0.654\n",
      "Step 1530 | Loss 0.650\n",
      "Step 1540 | Loss 0.659\n",
      "Step 1550 | Loss 0.647\n",
      "Step 1560 | Loss 0.639\n",
      "Step 1570 | Loss 0.668\n",
      "Step 1580 | Loss 0.680\n",
      "Step 1590 | Loss 0.659\n",
      "Step 1600 | Loss 0.664\n",
      "\n",
      "<start> was doing the same as every one did,\n",
      "laughed complacently and good-humoredly, while Levin laughed without\n",
      "complacency and sometimes angrily.\n",
      "\n",
      "\"We have long been expecting you,\" said Stepan Arkadyevit<end><pad>\n",
      "\n",
      "Step 1610 | Loss 0.661\n",
      "Step 1620 | Loss 0.641\n",
      "Step 1630 | Loss 0.645\n",
      "Step 1640 | Loss 0.635\n",
      "Step 1650 | Loss 0.657\n",
      "Step 1660 | Loss 0.630\n",
      "Step 1670 | Loss 0.640\n",
      "Step 1680 | Loss 0.656\n",
      "Step 1690 | Loss 0.664\n",
      "Step 1700 | Loss 0.647\n",
      "\n",
      "<start> the family, and the family, and Konstantin Levin love with the Shtcherbatskys' house, and\n",
      "he was in love with the Shtcherbatskys' house, and\n",
      "he was in love with the Shtcherbatskys' house, and\n",
      "he was <end>\n",
      "\n",
      "Step 1710 | Loss 0.642\n",
      "Step 1720 | Loss 0.654\n",
      "Step 1730 | Loss 0.639\n",
      "Step 1740 | Loss 0.632\n",
      "Step 1750 | Loss 0.642\n",
      "Step 1760 | Loss 0.622\n",
      "Step 1770 | Loss 0.660\n",
      "Step 1780 | Loss 0.626\n",
      "Step 1790 | Loss 0.651\n",
      "Step 1800 | Loss 0.639\n",
      "\n",
      "<start>the was in the Shtcherbatskys' house, and\n",
      "he was in love with the Shtcherbatsky household. Strange as it may\n",
      "appear, it was with the household, the family, that Konstantin Levin was\n",
      "in love, especiall<end>\n",
      "\n",
      "Step 1810 | Loss 0.632\n",
      "Step 1820 | Loss 0.639\n",
      "Step 1830 | Loss 0.618\n",
      "Step 1840 | Loss 0.631\n",
      "Step 1850 | Loss 0.633\n",
      "Step 1860 | Loss 0.648\n",
      "Step 1870 | Loss 0.636\n",
      "Step 1880 | Loss 0.639\n",
      "Step 1890 | Loss 0.627\n",
      "Step 1900 | Loss 0.622\n",
      "\n",
      "<start>he was in the Shtcherbatskys' house, and\n",
      "he was in love with the Shtcherbatskys' house, and\n",
      "he was in love with the Shtcherbatsky household. Strange as it may\n",
      "appear, it was with the household, the fa<end>\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1910 | Loss 0.620\n",
      "Step 1920 | Loss 0.615\n",
      "Step 1930 | Loss 0.611\n",
      "Step 1940 | Loss 0.604\n",
      "Step 1950 | Loss 0.612\n",
      "Step 1960 | Loss 0.623\n",
      "Step 1970 | Loss 0.612\n",
      "Step 1980 | Loss 0.595\n",
      "Step 1990 | Loss 0.622\n",
      "Step 2000 | Loss 0.642\n",
      "\n",
      "<start>the could not live with the Shtcherbatskys were old, noble Moscow\n",
      "families, and had always been on intimate and friendly terms. This\n",
      "intimacy had grown still closer during Levin's student days. He had<end>\n",
      "\n",
      "Step 2010 | Loss 0.634\n",
      "Step 2020 | Loss 0.613\n",
      "Step 2030 | Loss 0.610\n",
      "Step 2040 | Loss 0.608\n",
      "Step 2050 | Loss 0.618\n",
      "Step 2060 | Loss 0.599\n",
      "Step 2070 | Loss 0.603\n",
      "Step 2080 | Loss 0.599\n",
      "Step 2090 | Loss 0.600\n",
      "Step 2100 | Loss 0.581\n",
      "\n",
      "<start>in the professor, and he went\n",
      "back to his argument. \"No,\" he said; \"but said Sergey Ivanovitch, with his habitual\n",
      "clearness, precision of expression, and elegance of phrase. \"I cannnnot in\n",
      "the could n<end>\n",
      "\n",
      "Step 2110 | Loss 0.581\n",
      "Step 2120 | Loss 0.586\n",
      "Step 2130 | Loss 0.578\n",
      "Step 2140 | Loss 0.585\n",
      "Step 2150 | Loss 0.597\n",
      "Step 2160 | Loss 0.599\n",
      "Step 2170 | Loss 0.606\n",
      "Step 2180 | Loss 0.591\n",
      "Step 2190 | Loss 0.607\n",
      "Step 2200 | Loss 0.583\n",
      "\n",
      "<start>the conversation.\n",
      "\n",
      "A little man in spectacles, with a narrow forehead, tore himself from\n",
      "the discussion for an instant to greet Levin, and then went on talking\n",
      "without paying any further attention to <end>\n",
      "\n",
      "Step 2210 | Loss 0.584\n",
      "Step 2220 | Loss 0.576\n",
      "Step 2230 | Loss 0.590\n",
      "Step 2240 | Loss 0.596\n",
      "Step 2250 | Loss 0.593\n",
      "Step 2260 | Loss 0.587\n",
      "Step 2270 | Loss 0.614\n",
      "Step 2280 | Loss 0.613\n",
      "Step 2290 | Loss 0.580\n",
      "Step 2300 | Loss 0.577\n",
      "\n",
      "<start>the conversation.\n",
      "\n",
      "A little man in spectacles, with a narrow forehead, tore himself from\n",
      "the discussion for an instant to greet Levin, and then went on talking\n",
      "without paying any further attention to <end>\n",
      "\n",
      "Step 2310 | Loss 0.595\n",
      "Step 2320 | Loss 0.583\n",
      "Step 2330 | Loss 0.587\n",
      "Step 2340 | Loss 0.568\n",
      "Step 2350 | Loss 0.592\n",
      "Step 2360 | Loss 0.594\n",
      "Step 2370 | Loss 0.583\n",
      "Step 2380 | Loss 0.573\n",
      "Step 2390 | Loss 0.598\n",
      "Step 2400 | Loss 0.583\n",
      "\n",
      "<start> the professor, he expression, sence yof the could not and not the metings.\"\n",
      "\n",
      "\"Well, how is you cou do skate Prokates sof him in that he ice. \"I really don't know.\"\n",
      "\n",
      "\"What! Why, sured Sergey Ivanovitc<end>\n",
      "\n",
      "Step 2410 | Loss 0.580\n",
      "Step 2420 | Loss 0.576\n",
      "Step 2430 | Loss 0.586\n",
      "Step 2440 | Loss 0.587\n",
      "Step 2450 | Loss 0.593\n",
      "Step 2460 | Loss 0.585\n",
      "Step 2470 | Loss 0.594\n",
      "Step 2480 | Loss 0.587\n",
      "Step 2490 | Loss 0.556\n",
      "Step 2500 | Loss 0.564\n",
      "\n",
      "<start> on the point of starting off at once.\n",
      "\n",
      "\"I am sorry I told you,\" said Sergey Ivanovitch.\n",
      "\"As regards myself, I have no fear of your doing so; he will not make\n",
      "you quarrel with me; but for your own sak<end>\n",
      "\n",
      "Step 2510 | Loss 0.557\n",
      "Step 2520 | Loss 0.561\n",
      "Step 2530 | Loss 0.567\n",
      "Step 2540 | Loss 0.571\n",
      "Step 2550 | Loss 0.570\n",
      "Step 2560 | Loss 0.559\n",
      "Step 2570 | Loss 0.557\n",
      "Step 2580 | Loss 0.557\n",
      "Step 2590 | Loss 0.565\n",
      "Step 2600 | Loss 0.565\n",
      "\n",
      "<start> the smile of the skate. \"Except you,\n",
      "there's none of the could, herre and the smiliars.\n",
      "\n",
      "\"Oh, yes, yes; make haste, please,\" answered Levin, with difficulty\n",
      "restraining the smile of rapture which wou<end>\n",
      "\n",
      "Step 2610 | Loss 0.549\n",
      "Step 2620 | Loss 0.570\n",
      "Step 2630 | Loss 0.565\n",
      "Step 2640 | Loss 0.568\n",
      "Step 2650 | Loss 0.560\n",
      "Step 2660 | Loss 0.542\n",
      "Step 2670 | Loss 0.544\n",
      "Step 2680 | Loss 0.556\n",
      "Step 2690 | Loss 0.535\n",
      "Step 2700 | Loss 0.553\n",
      "\n",
      "<start>the pavilion with Mlle. Linon, and looked towards him with a\n",
      "smile of quiet affection, as though he were a favorite brother. \"And can\n",
      "it be my fault, can I have done anything wrong? They talk of flirt<end>\n",
      "\n",
      "Step 2710 | Loss 0.549\n",
      "Step 2720 | Loss 0.578\n",
      "Step 2730 | Loss 0.560\n",
      "Step 2740 | Loss 0.571\n",
      "Step 2750 | Loss 0.555\n",
      "Step 2760 | Loss 0.561\n",
      "Step 2770 | Loss 0.563\n",
      "Step 2780 | Loss 0.566\n",
      "Step 2790 | Loss 0.557\n",
      "Step 2800 | Loss 0.563\n",
      "\n",
      "<start> that he English\n",
      "nursery tale. \"Do you remember that's what you used to call them?\"\n",
      "\n",
      "He remembered absoluted the more the boween ards the beginnning of the\n",
      "winter.\n",
      "\n",
      "\"Are you going to stay in town long<end>\n",
      "\n",
      "Step 2810 | Loss 0.559\n",
      "Step 2820 | Loss 0.569\n",
      "Step 2830 | Loss 0.558\n",
      "Step 2840 | Loss 0.565\n",
      "Step 2850 | Loss 0.546\n",
      "Step 2860 | Loss 0.546\n",
      "Step 2870 | Loss 0.546\n",
      "Step 2880 | Loss 0.546\n",
      "Step 2890 | Loss 0.540\n",
      "Step 2900 | Loss 0.530\n",
      "\n",
      "<start>ing that the was\n",
      "holding him in check by her composed tone, which he would not have the\n",
      "force to break through, just as it had been at the beginning of the\n",
      "winter.\n",
      "\n",
      "\"Are you going to stay in town long<end>\n",
      "\n",
      "Step 2910 | Loss 0.540\n",
      "Step 2920 | Loss 0.540\n",
      "Step 2930 | Loss 0.541\n",
      "Step 2940 | Loss 0.527\n",
      "Step 2950 | Loss 0.533\n",
      "Step 2960 | Loss 0.533\n",
      "Step 2970 | Loss 0.532\n",
      "Step 2980 | Loss 0.518\n",
      "Step 2990 | Loss 0.534\n",
      "Step 3000 | Loss 0.536\n",
      "\n",
      "<start> of the day,\n",
      "came out of the pavilion with Mlle. Linon, and looked towards him with a\n",
      "smile of quiet affection, as though he were a favorite brother. \"And can\n",
      "it be my fault, can I have done anything <end>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
